import UrlDb;

import os;
import time;
import threading;
import logging;


class StreamParser:
    def __init__(self, streamFile, elementParser, urlConverter, config):
        print 'StreamParser::__init__()';
        taskId = config.taskId;
        self.toCrawlHomeUrlFilePath = config.toCrawlHomeUrlFilePath;
        self.toCrawlHomeUrlFileLock = config.toCrawlHomeUrlFileLock;
        
        self.streamFile = streamFile;
        self.elementParser = elementParser;
        self.urlConverter = urlConverter;
        
        self.streamFilePath = config.streamFilePath;
        self.streamMetaFilePath = os.path.join(taskId, 'stream.meta');
        self.streamMetaFileLock = threading.Lock();
        self.postingUrlDb =  UrlDb.UrlDb(os.path.join(taskId, 'posting.url.db'));
        self.postingUrlDbLock = threading.Lock();
        self.postingFilePath = os.path.join(taskId, 'posting.dat');
        self.postingFileLock = threading.Lock();
        self.postingUrlFilePath = os.path.join(taskId, 'posting.url.dat');
        self.postingUrlFileLock = threading.Lock();
        
        
    def parse(self):
        while 1:
            elements = self.readStream(1000);
            print time.asctime(),': StreamParser::parse() have read', elements.__len__(),'elements';
            results = [];
        
            if not len(elements):
                time.sleep(5);
                continue;
        
#            print time.time(), ': now parsing';
            for element in elements:
                homeUrl, posting = self.elementParser.parse(element);
                if homeUrl:
                    results.append((homeUrl, posting));
            print time.asctime(),": StreamParser::parse() have processed", results.__len__(), 'elements';
                    
            self.postingUrlDbLock.acquire();
            self.postingFileLock.acquire();
            self.postingUrlFileLock.acquire();
            self.toCrawlHomeUrlFileLock.acquire();
            postingFile = open(self.postingFilePath, 'a');
            print time.asctime(), ': StreamParser::parse() acquired locks and opened', self.postingFilePath;
            postingUrlFile = open(self.postingUrlFilePath, 'a');
            toCrawlHomeUrlFile = open(self.toCrawlHomeUrlFilePath, 'a');
            cnt = 0;
            for result in results:
                homeUrl = result[0];
                posting = result[1];
                if not self.postingUrlDb.contains(posting.url):                    
                    postingStr = str(posting);
                    if postingStr:
                        self.postingUrlDb.add(homeUrl);
                        postingFile.write(str(posting) + '\n');
                        postingUrlFile.write(posting.url + '\n');
                        homeUrl = self.urlConverter.posting2Home(posting.url);
                        cnt += 1;
                    else:
                        logging.warn("ERROR in parsing " + posting.url);
                    toCrawlHomeUrlFile.write(homeUrl + '\n');
            print time.asctime(),': StreamParser::parse() adding',cnt,'postings';
            toCrawlHomeUrlFile.close();
            postingFile.close();
            postingUrlFile.close();
            self.toCrawlHomeUrlFileLock.release();
            self.postingUrlFileLock.release();
            self.postingFileLock.release();
            self.postingUrlDbLock.release();

        
    def readStream(self, maxNum):
        #1. get position 
        print time.asctime(),': StreamParser::readStream() starting';
        self.streamMetaFileLock.acquire();
        position = 0;
        if os.path.exists(self.streamMetaFilePath):
            metaFile = open(self.streamMetaFilePath);
            position = int(metaFile.readline().strip());
            metaFile.close();
    
        #2. read element from stream file
        streamFile = self.streamFile;
        elements = [];
        if streamFile.open(self.streamFilePath) < 0:
            logging.warn('StreamParser::readStream: Cant open stream file', self.streamFilePath);
            print 'return'
            return elements;
        streamFile.seek(position);
        elementNum = 0;
        while 1:
            self.streamFile.next();
            element = self.streamFile.getCurrElement();
            if not element or elementNum >= maxNum:
                break;
            elements.append(element);
            elementNum += 1;
            position = self.streamFile.getCurrPos();
        streamFile.close();
            
        #3. restore position
        metaFile = open(self.streamMetaFilePath, 'w');
        metaFile.write(str(position));
        metaFile.close();
        self.streamMetaFileLock.release();
        
        return elements;